package com.shyroke.daydayzhuansvc.util;

import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.StrUtil;
import com.google.common.base.Joiner;
import com.google.gson.Gson;
import com.shyroke.daydayzhuanapi.constant.CommitConstant;
import com.shyroke.daydayzhuanapi.constant.ProjectConst;
import com.shyroke.daydayzhuanapi.dto.*;
import com.shyroke.daydayzhuanapi.entity.LabelEntity;
import com.shyroke.daydayzhuanapi.entity.ProjectEntity;
import com.shyroke.daydayzhuanapi.entity.SpiderInfoEntity;
import com.shyroke.daydayzhuanapi.entity.TypeEntity;
import com.shyroke.daydayzhuansvc.service.impl.*;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.CollectionUtils;

import java.net.URL;
import java.net.URLDecoder;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.stream.Collectors;

/**
 * 类名 ：爬虫线程
 * 用法 ：
 * 创建人 ： shyroke
 * 时间：2019/2/22 13:23
 */
public class SpiderRunnable implements Callable {

    private Logger logger = LoggerFactory.getLogger(this.getClass());

    private SpiderInfoEntity spiderInfo;
    private String active;

    public SpiderRunnable(SpiderInfoEntity spiderInfo, String active){
        this.spiderInfo = spiderInfo;
        this.active = active;
    }

    @Override
    public Object call() {
        SpiderService spiderService =   ApplicationContextProvider.getBean(SpiderService.class);
        CommitService commitService =   ApplicationContextProvider.getBean(CommitService.class);
        OSSService ossService =   ApplicationContextProvider.getBean(OSSService.class);
        ProjectService projectService =   ApplicationContextProvider.getBean(ProjectService.class);
        LableService lableService =   ApplicationContextProvider.getBean(LableService.class);
        TypeService typeService =   ApplicationContextProvider.getBean(TypeService.class);

        logger.info("======="+spiderInfo.getSiteName()+"线程=======");
        try {
            String todayTime = "";
            if("true" != spiderInfo.getDateIgnore()){
                 todayTime = CommonUtil.getDateByFormat(new Date(),spiderInfo.getTodayFormat());
            }

            //文章详情中需要排除的标签,比如广告之类的
            List<String> descExcludeCssQuery = new ArrayList<>();
            if(!StringUtils.isBlank(spiderInfo.getDescExcludeCssQuery())){
                String[] descStrArr = spiderInfo.getDescExcludeCssQuery().split(",");
                for(String desc : descStrArr){
                    descExcludeCssQuery.add(desc);
                }
            }

            if(StringUtils.isBlank(spiderInfo.getDescUrlPre())){
                spiderInfo.setDescUrlPre(null);
            }

            boolean dateIgnore = false;
            if(!StringUtils.isBlank(spiderInfo.getDateIgnore()) && spiderInfo.getDateIgnore().equals("true")){
                dateIgnore = true;
            }

            List<SpiderProjectDto> spiderProjects = spiderService.spiderSite(spiderInfo.getSiteName(),spiderInfo.getSiteUrl(),spiderInfo.getArticleEleCssQuery(),todayTime,
                    spiderInfo.getDateCssQuery(),spiderInfo.getTitleCssQuery(),spiderInfo.getUrlCssQuery(),spiderInfo.getDescCssQuery(),descExcludeCssQuery,spiderInfo.getDescUrlPre(),dateIgnore);

            Map<String, com.shyroke.daydayzhuanapi.dto.SpiderProjectDto> stringSpiderProjectMap = new HashMap<>();
            List<com.shyroke.daydayzhuanapi.dto.SpiderProjectDto> existProjectList = spiderService.findByProjectNameList(spiderProjects);
            if(!CollectionUtils.isEmpty(existProjectList)){
                for(com.shyroke.daydayzhuanapi.dto.SpiderProjectDto spiderProject:existProjectList){
                    stringSpiderProjectMap.put(spiderProject.getTitle(),spiderProject);
                }
            }
            List<SpiderProjectDto> saveProjectList = new ArrayList<>();
            if(!CollectionUtils.isEmpty(spiderProjects)){
                for(SpiderProjectDto project:spiderProjects){
                    com.shyroke.daydayzhuanapi.dto.SpiderProjectDto existProject = stringSpiderProjectMap.get(project.getTitle());
                    if(existProject == null){
                        if(spiderInfo.getSiteUrl().contains("zhengyuw.com")){
                            if(project.getTitle().startsWith("【")){
                                project.setTitle(project.getTitle().substring(project.getTitle().indexOf("】")+1));
                            }else if(project.getTitle().startsWith("[")){
                                project.setTitle(project.getTitle().substring(project.getTitle().indexOf("]")+1));
                            }
                        }
                        if(spiderInfo.getSiteUrl().contains("llxbw.com")){
                            String contents = project.getContent().replaceAll("乐乐线报", "天天线报");
                            project.setContent(contents);
                        }
                        project.setIsSave(0);
                        project.setOnlyWrite(0);
                        project.setTitle(EmojiUtil.filterEmoji(project.getTitle()));
                        project.setContent(EmojiUtil.filterEmoji(project.getContent()));
                        saveProjectList.add(project);
                    }
                }
            }

            //获取当天的项目名集合
            List<ProjectDto> currenProjectList = projectService.getCurrentProjectList();
            List<String> projectNameList = new ArrayList<>();
            for (ProjectDto projectDto : currenProjectList) {
                String projectName = projectDto.getProjectName();
                if(StringUtils.isEmpty(projectName)){
                    continue;
                }
                String[] projectNameArr = projectName.split(",");
                projectNameList.addAll(Arrays.asList(projectNameArr));
            }

            if(spiderInfo.getAutoSave() != null && spiderInfo.getAutoSave() == 1){
                //自动保存入库
                if(!CollectionUtils.isEmpty(saveProjectList)){
                    List<TypeDto> allTypeList = typeService.findAll();
                    allTypeList.stream().forEach(typeDto -> {
                        if(StrUtil.isNotBlank(typeDto.getLabelIds())) {
                            List<Integer> labelIdList = Arrays.stream(typeDto.getLabelIds().split(",")).map(ids -> Integer.valueOf(ids)).collect(Collectors.toList());
                            List<LabelEntity> labelList = lableService.findByIdIn(labelIdList);
                            typeDto.setLabelList(labelList);
                        }
                    });
                    
                    for(SpiderProjectDto spiderProject:saveProjectList){
                        try {
                            ProjectDto project = coverProejct(commitService,spiderProject,ossService,lableService,spiderInfo,allTypeList,active);
                            if(project == null){
                                continue;
                            }
                            boolean isExist = false;
                            for (String name : projectNameList) {
                                if(project.getTitle().contains(name)){
                                    isExist = true;
                                    break;
                                }
                            }
                            if(isExist){
                                continue;
                            }
                            Integer projectId = projectService.save(project);
//                            if(!"dev".equals(active)){
//                                String content = Jsoup.parse(project.getContent()).text();
//                                List<String> urlList = new ArrayList<>();
//                                urlList.add("https://www.daydayzhuan.com/article/"+projectId);
//                                if(content.length() > 400){
//                                    //大于400个字走快速收录
//                                    logger.info("=================自动提交百度快速收录，url="+urlList.get(0));
//                                    String result = commitService.commitBaidu(CommitConstant.BAIDU_FAST_COMMIT_URL,urlList);
//                                    logger.info("============"+result+"===========");
//                                }else{
//                                    logger.info("=================自动提交百度普通收录，url="+urlList.get(0));
//                                    String result = commitService.commitBaidu(CommitConstant.BAIDU_COMMIT_URL,urlList);
//                                    logger.info("============"+result+"===========");
//                                }
//                                logger.info("=================自动提交Bing收录，url="+urlList.get(0));
//                                String result = commitService.commitBing(CommitConstant.BING_COMMIT_URL,urlList);
//                                logger.info("============"+result+"===========");
//                            }
                            //设置为已保存
                            spiderProject.setIsSave(1);
                        }catch (Exception e){
                            logger.info("自动保存入库异常",e);
                            continue;
                        }
                    }
                }
            }

            spiderService.batchSave(saveProjectList);
        }catch (Exception e){
            logger.info(spiderInfo.getSiteName()+"线程异常，原因="+ExceptionUtil.getMessage(e));
            return R.error(ExceptionUtil.getMessage(e));
        }
        logger.info("======="+spiderInfo.getSiteName()+"线程结束=======");
        return R.ok(spiderInfo.getSiteName()+"爬取成功！");
    }

    private ProjectDto coverProejct(CommitService commitService, SpiderProjectDto spiderProject, OSSService ossService, LableService lableService, SpiderInfoEntity spiderInfo, List<TypeDto> allTypeList, String active) throws Exception{
        ProjectDto project = new ProjectDto();
        //设置标题
        project.setTitle(spiderProject.getTitle());
        //设置摘要
        String text = Jsoup.parse(spiderProject.getContent()).text();
        if(StringUtils.isNotEmpty(text)){
            if(text.length() > 108){
                project.setSummary(text.substring(0,108));
            }else{
                project.setSummary(text.substring(0,text.length()));
            }
        }

        //伪原创文章内容
        String machineContent = null;
        if(spiderInfo.getIsMachine() == 1 && !"dev".equals(active)){
            machineContent = commitService.commit5118(spiderProject.getContent());
            project.setContent(URLDecoder.decode(new Gson().fromJson(machineContent, MachineRewrite5118Resp.class).getData().replaceAll("%(?![0-9a-fA-F]{2})", "%25"),"utf-8"));
        }else{
            project.setContent(spiderProject.getContent());
        }
        //设置type和label
        Map<String,String> type = getSpiderArticleType(project.getTitle(), allTypeList);
        if(type == null){
            return null;
        }
        project.setTypes(type.get("id"));
        project.setKeyword(type.get("typeName"));
        project.setCommented("0");
        project.setIsNofollow(0);
        String sourceURL = "";
        String source = spiderProject.getSource();
        if(!org.springframework.util.StringUtils.isEmpty(source)){
            String[] sourceArr = source.split(",");
            sourceURL = sourceArr[1];
        }
        project.setSourceURL(sourceURL);
        //上传图片到自己的oss
        try {
            ossService.handleImg(project,spiderInfo.getSiteUrl().contains("yyok")?spiderProject.getUrl():spiderInfo.getSiteUrl());
        }catch (Exception e){
            return null;
        }
         //删除a标签中的网址
        Document document = Jsoup.parse(project.getContent());
        Elements Aelements = document.select("a");
        for(Element aElement:Aelements){
            aElement.attr("rel","nofollow");
            String href = aElement.attr("href");
            if(!spiderInfo.getSiteUrl().contains("yyok")){
                if(href.contains(new URL(sourceURL).getHost())){
                    aElement.removeAttr("href");
                    aElement.tagName("span");
                }
            }
        }
        project.setContent(document.html());

        //默认不在首页显示
        project.setIsHomePage("1");
        project.setType(ProjectConst.NORMAL_PROJECT); //标志为普通项目
        project.setCreateTime(DateUtils.sub(new Date(),1)); //设置创建时间
        project.setReadMount(0);//阅读数量
        project.setCommentMount(0);//评论数量
        project.setCreateId(1);//创建人ID
        project.setOffline(ProjectConst.UNOFFLINE); //默认未下架
        project.setUpdateTime(DateUtils.sub(new Date(),1));
        project.setCommented("0");

        return project;
    }

    /**
     * 根据标题获取label集合
     * @param title
     * @param allLabelList
     * @return
     */
    private String getSpiderArticleLabels(String title, List<LabelDto> allLabelList, ProjectDto project) {
       Set<String> keyList = new HashSet<>();
        Set<String> keyIdList = new HashSet<>();
        for (LabelDto label : allLabelList) {
            String keys = label.getKeys();
            if(StringUtils.isEmpty(keys)){
                continue;
            }
            String[] keyArr = keys.split(",");
            for (String key : keyArr) {
                if(title.contains(key)){
                    keyList.add(label.getLabelName());
                    keyIdList.add(label.getId().toString());
                }
            }
        }

        project.setLabels(Joiner.on(",").join(keyIdList));
        return Joiner.on(",").join(keyList);
    }

    /**
     * 根据标题获取type
     * @param allTypeList
     * @param title
     * @return
     */
    private Map<String,String> getSpiderArticleType(String title,List<TypeDto> allTypeList) {
        Map<String,String> res = null;
        a:for (TypeDto type : allTypeList) {
            List<LabelEntity> labelList = type.getLabelList();
            if(CollUtil.isNotEmpty(labelList)){
                for (LabelEntity label : labelList) {
                    String keys = label.getKeys();
                    if(StrUtil.isNotBlank(keys)) {
                        keys = keys.replaceAll("，", ",");
                        String[] keyArr = keys.split(",");
                        if(keyArr != null){
                            List<String> keyList = Arrays.asList(keyArr);
                            for (String key : keyList) {
                                if(title.contains(key)){
                                    res = new HashMap<>();
                                    res.put("id",type.getId().toString());
                                    res.put("typeName",label.getLabelName());
                                    break a;
                                }
                            }
                        }
                    }
                }
            }
        }
        return res;
    }
}
